/**
 *  抓取点评热点店铺
 *  
 */ 
const puppeteer = require('puppeteer')
const path = require('path')
const fs = require('fs') 
const { URL } = require('url');


const numCPUs = require('os').cpus().length; 

const link = 'http://www.dianping.com/'
//需要抓取的城市数据
const need_fetch_cities = [ { name:'chengdu',url: 'http://www.dianping.com/shoplist/search/8_10_0_score '},{
  name: 'beijing',url: 'http://www.dianping.com/shoplist/search/2_10_0_score'
},  { name: 'guangzhou' ,url: 'http://www.dianping.com/shoplist/search/4_10_0_score'},
{ name: 'shenzhen', url: 'http://www.dianping.com/shoplist/search/7_10_0_score'},]
 
 
puppeteer.launch({headless: false}).then(async browser => {
  browser.on('targetcreated', _=> {
    console.log(_);
    
  })

  for (const city of need_fetch_cities) {
    var p = path.join(process.cwd(), 'cities1', `${city.name}.json`);
    if (!fs.existsSync(p)) {
      const writeStream = fs.createWriteStream(p);
      const page = await browser.newPage();    
      let callback = (res) => {
        if (res.ok) {
          var req = res.request();          
          if (req.url.startsWith('http://www.dianping.com/mylist/ajax/shoprank')) {
            console.log(req.url);
            res.json().then((json) => {
              console.log(json);
              writeStream.write(JSON.stringify(json));
              writeStream.end();
            });
          }
        }   
      }

      await page.goto(city.url);
      await page.waitFor(500);
      await page.click('#Top > p:nth-child(2) > a:nth-child(2)');
      page.on('response', callback);
      await page.waitFor(1000);  
    }
  }

});

async function excute(shopUrl, browser, callback) {
  const page = await browser.newPage();
  page.on('response', callback);
  await page.goto(shopUrl);
  await sleep(5000);
}
function sleep(ms) {
  return new Promise(resolve => setTimeout(resolve, ms))
}

 
